﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using HtmlAgilityPack;
using System.IO;
using System.Net;
using System.Drawing;
using System.Threading;
using NLog;

namespace LingDong.NaiveCrawler
{
    public class PictureCrawl
    {
        public static void CrawlHTML(string content, string file, string outputImageDir)
        {
            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(content);
            HtmlNodeCollection imageNodeList = doc.DocumentNode.SelectNodes("//img[@src]");
            if (imageNodeList == null)
                return;

            using (StreamWriter sw = new StreamWriter(outputImageDir + "index.txt", true))
            {
                foreach (HtmlNode imageNode in imageNodeList)
                {
                    SaveImageNode(imageNode, outputImageDir, file, sw);

                    Thread.Sleep(500);
                }                
            }
        }

        private static void SaveImageNode(HtmlNode imageNode, string outputImageDir, string file, StreamWriter indexSW)
        {
            string src = imageNode.Attributes["src"].Value.Trim();
            if (!IsSrcImage(src))
                return;

            // declared size is too small
            int width = 0;
            if (imageNode.Attributes.Contains("width"))
                Int32.TryParse(imageNode.Attributes["width"].Value, out width);
            int height = 0;
            if (imageNode.Attributes.Contains("height"))
                Int32.TryParse(imageNode.Attributes["height"].Value, out height);
            if (width * height != 0 && width * height <= IgnoreImageSize)
                return;

            if (downloadedImage.ContainsKey(src))
                return;

            WebClient client = new WebClient();
            string tempFile = Path.GetTempFileName();
            try
            {
                client.DownloadFile(src, tempFile);
            }
            catch (Exception)
            {
                logger.Error(src);
                return;
            }
            int size = 0;
            using (Image pic = Image.FromFile(tempFile))
            {
                size = pic.Height * pic.Width;
            }
            if (size <= IgnoreImageSize)
            {
                File.Delete(tempFile);
                return;
            }

            downloadedImage[src] = 1;
            string imageFile = String.Format("{0}{1}{2}", outputImageDir, imageCounter++, Path.GetExtension(src));
            File.Move(tempFile, imageFile);
            indexSW.WriteLine("{0}\t{1}\t{2}", file, src, imageFile);
            indexSW.Flush();
        }

        private static bool IsSrcImage(string src)
        {
            foreach (string name in ImageLegalExt)
            {
                if (src.EndsWith(name))
                    return true;
            }
            return false;
        }

        private static int imageCounter = 1;
        private static List<string> ImageLegalExt = new List<string>() { "jpg", "gif", "png", "jpge" };
        private static int IgnoreImageSize = 30000;
        private static Logger logger = LogManager.GetCurrentClassLogger();

        private static Dictionary<string, int> downloadedImage = new Dictionary<string, int>();
    }
}
